{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ms_jw = sorted(glob('google-translate-ms-jw/*requested'))\n",
    "len(ms_jw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('processed-ms-jw.jsonl', 'w') as fopen_l:\n",
    "    for f in ms_jw:\n",
    "        with open(f) as fopen:\n",
    "            for l in fopen:\n",
    "                l = json.loads(l)\n",
    "                try:\n",
    "                    left = l['src']\n",
    "                    right = l['r']['result']\n",
    "                    d = {\n",
    "                        'left': left,\n",
    "                        'right': right,\n",
    "                    }\n",
    "                    fopen_l.write(f'{json.dumps(d)}\\n')\n",
    "                except:\n",
    "                    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1683631 processed-ms-jw.jsonl\r\n"
     ]
    }
   ],
   "source": [
    "!wc -l processed-ms-jw.jsonl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"left\": \"Eyyy WTF\", \"en\": \"Hey, what the hell?\", \"ms\": \"Eh, apa ni?\"}\r\n",
      "{\"left\": \"i know plenty of people who snack on sambal ikan bilis.\", \"en\": \"I know many people who snack on sambal ikan bilis.\", \"ms\": \"Saya kenal ramai orang yang suka makan snek sambal ikan bilis.\"}\r\n",
      "{\"left\": \"I often visualize my own programs algorithm before implemment it.\", \"en\": \"I frequently visualise the algorithm of my own programs before implementing them.\", \"ms\": \"Saya kerap memvisualisasikan algoritma program saya sebelum melaksanakannya.\"}\r\n",
      "{\"left\": \"YASSSSSSS\", \"en\": \"Yes\", \"ms\": \"Ya\"}\r\n",
      "{\"left\": \"Am I the only one who used their given name ever since I was a kid?\", \"en\": \"Did anyone else use their given name since they were a child?\", \"ms\": \"Adakah saya satu-satunya yang menggunakan nama yang diberikan sejak saya kecil lagi?\"}\r\n",
      "{\"left\": \"*JENGKAUT* has entered the chat.\", \"en\": \"*JENGKAUT* has entered the chat.\", \"ms\": \"*JENGKAUT* telah masuk ke dalam perbualan.\"}\r\n",
      "{\"left\": \"I think I won't be able to catch it within 1 min lol\", \"en\": \"I don't think I'll be able to catch it within one minute, haha\", \"ms\": \"Saya rasa saya tidak akan dapat menangkapnya dalam masa satu minit, haha\"}\r\n",
      "{\"left\": \"Smh the dude literally has a rubbish bin infront of his house\", \"en\": \"Shaking my head, the man has a garbage bin in front of his house.\", \"ms\": \"Goyang kepala, lelaki itu mempunyai tong sampah di depan rumahnya.\"}\r\n",
      "{\"left\": \"It's all good, I know it's the default gender for any figure of speech. Just need to broadcast to actual sexists out there that philosophy and reason are not inherent to only men.\", \"en\": \"It's all fine, I understand that using the male gender as a default in figures of speech is common. However, it's important to make it clear to any actual sexists out there that intelligence and rational thinking are not exclusive to men.\", \"ms\": \"Semuanya baik-baik saja, saya tahu bahawa penggunaan gender lelaki sebagai default dalam ungkapan adalah biasa. Namun, penting untuk menyampaikan kepada sesiapa yang sebenarnya berfikiran seksis bahawa kebijaksanaan dan pemikiran rasional tidak terbatas hanya pada lelaki sahaja.\"}\r\n",
      "{\"left\": \"Gotta be wary of pimples. Oh they bleed bad when cut\", \"en\": \"We must be cautious of pimples. They tend to bleed heavily when cut.\", \"ms\": \"Kita perlu berhati-hati dengan jerawat. Mereka cenderung berdarah banyak apabila dipotong.\"}\r\n"
     ]
    }
   ],
   "source": [
    "!tail -n 10 process-manglish.jsonl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
